#-*-coding:utf-8-*-
from selenium import webdriver
from lxml import etree
import time
import re
import threading
import codecs
import MySQLdb
import json
import requests
#将获取到的数据保存到数据库
#conn = MySQLdb.connect(host='127.0.0.1',user = 'root',passwd = 'root',db = 'jd',charset = 'utf8')
#print conn
#cur = conn.cursor()
#sql = "SELECT * FROM jds"
#print cur.execute(sql)

#京东商品采用反爬虫，价格是使用js加载进来的，所以，使用无头浏览器phantomjs加载网页获取源代码
def by_url_get_all(url):
	print "正在获取：%s 的网页代码..."%url
	try:
		#启动phantomjs浏览器
		browser = webdriver.PhantomJS(executable_path="C:\Python27\phantomjs\\bin\phantomjs.exe")
		browser.get(url)
		browser.implicitly_wait(30)
		html = browser.page_source	
		#使用玩之后，关闭，不然占用太多内存导致电脑死机
		browser.quit()
		#返回网页代码
		return html
	except:
		print "获取网页数据失败！！！"
#保存获取到的数据
def write_and_save(title,link,sku_id,price,comnum,shop_name,shop_grade):
	#首先判断提取到了商品标题。如果没有商品名称则不保存
	if len(title) != 0 :
		titles = title[0][:-16]
		links = link.replace('\n','')
		sku_ids = sku_id
		if len(price)==0:
			prices = "None"
		else :
			prices = price[0][2:-7]

		if len(comnum)==0:
			comnums = "None"
		else :
			comnums = comnum[0]

		if len(shop_name)==0:
			shop_names = "None"
		else :
			shop_names = shop_name[0][23:-49]

		if len(shop_grade)==0:
			shop_grades = "None"
		else :
			shop_grades = shop_grade[0]
		print "(%s,%s,%s,%s,%s,%s,%s)"%(titles,links,sku_ids,prices,comnums,shop_names,shop_grades)
		#写入到json文件
		try:
			f = codecs.open('E:\jiaocheng\F\Project\jd.json', 'a', encoding="utf-8")
			goods = {"title":titles,"link":links,"sku_id":sku_ids,"price":prices,"comnum":comnums,"shop_name":shop_names,"shop_grade":shop_grades}
			#sql = "INSERT INTO jds(title,link,sku_id,price,comnum,shop_name,shop_grade)	VALUES(%s,%s,%s,%s,%s,%s,%s)"#%(titles,links,sku_ids,prices,comnums,shop_names,shop_grades)
			i = json.dumps(dict(goods),ensure_ascii=False)
			value = i + '\n'
			f.write(value)
			f.close()
			#cur.execute(sql,value)
			#conn.commit()
			print "Success--"
		except:
			print "Error---"
			#conn.rollback()
		return True
	else :
		return False

#从网页代码中提取需要的数据的类
class Gte_content(threading.Thread):
	def __init__(self,url):
		threading.Thread.__init__(self)
		self.url = url
	#主函数
	def run(self):
		try:
			#根据url调用函数并获取网页代码
			html = by_url_get_all(self.url)
			#使用lxml解析网页	
			tree = etree.HTML(html)
			#提取标题，就是商品名称
			title = tree.xpath("//head/title/text()")
			#正则匹配商品价格
			pr = re.compile(r'<span class="price J-p-[0-9]{1,13}">[0-9]{1,7}\.[0-9]{2}</span>')
			price1 = re.findall(pr,html)
			if len(price1)!= 0:
				pr2 = re.compile(r'">[0-9]{1,7}\.[0-9]{2}</span>')
				price = re.findall(pr2,price1[0])
			else:
				price = []
			#店铺名称
			sg = re.compile(r'target="_blank" title=".*?" clstag="shangpin\|keycount\|product\|dianpuname1">')
			shop_name = re.findall(sg,html)
			#店铺评分
			shop_grade = tree.xpath('//*[@id="crumb-wrap"]/div/div[2]/div[2]/div[2]/div/div/em/span/a/text()')
			#商品评论数
			comnum = tree.xpath('//*[@id="comment-count"]/a/text()')
			#商品链接
			link = self.url
			#商品id号
			sku_id = self.url[20:-5]
		except:
			print "Get_Page_html_Failure"
		try:
			
			#保存提取到的数据
			TF = write_and_save(title,link,sku_id,price,comnum,shop_name,shop_grade)
		except:
			print "write_error"

#获取该分类下的页码数，返回最大页码数字
def get_tag_page(url):
	print url
	try:
		html = requests.get(url)
		tree = etree.HTML(html.content)
		#提取页码
		page = tree.xpath("//span[@class='p-num']/a[last()-1]/text()")
		return int(page[0])
	except:
		return int(1)
#主函数
def main():
	#创建线程列表
	thread = []

	#用来测试的分类链接url1以及url2
	#url1=========数码相机
	#url2=========智能手环
	url1 = 'https://list.jd.com/list.html?cat=652,654,831'
	url2 = 'https://list.jd.com/list.html?cat=652,12345,12347'
	try:
		#获取该分类下商品页码数
		page = get_tag_page(url1)
		print page
		#需要抓取的商品url链接列表
		urllist = []
		#遍历所有的页面页码
		for i in range(1,page+1):
			urls = url1 + '&page=' + str(i)
			html = requests.get(urls)
			htmls = html.content
			#使用正则表达式过滤商品url链接
			com = re.compile(r'//item\.jd\.com/[0-9]{7,13}\.html')
			fin = re.findall(com,htmls)
			#遍历所有找到的商品url链接
			for url in fin:
				#判断链接是否重复，如果之前没有处理过，则添加到列表中
				if url in urllist:
					print '/////'
				else:
					print url
					urllist.append('https:' + url)
	except:
		print "商品链接获取失败！！！"
	print "共获取到：%d 个商品链接！"%len(urllist)
	#num为创建同时开启的线程数，默认使用1个线程
	num = 1
	urllist = list(set(urllist))
	print "经过筛选去重，共获取到：%d 个商品链接！"%len(urllist)
	try:
		#遍历之前提取到的商品链接urllist
		for i in range(0,len(urllist),num):
			#创建num个线程
			for j in range(i,i+num):
				t =  Gte_content(urllist[i + j])
				thread.append(t)
			#开启线程 

			for k in range(i,i+num):
				thread[k].start()
			#等待线程结束
			for m in range(i,i+num):
				thread[m].join()
			print "threading--ending------"
			time.sleep(1)
	except:
		print "线程创建失败！！！"
if __name__ == '__main__':
	main()

#如果需要抓取京东所有几百万商品的信息数据，在main函数中嵌入以下函数，即可
"""
#获取京东商品所有分类的分类链接
def get_tag_link():
	#所有的分类的链接在url中可以提取到 
	url = "https://www.jd.com/allSort.aspx"
	url_lists = []
	html = requests.get(url)
	#使用正则过滤所有的分类链接
	com = re.compile(r'<a href="//list\.jd\.com/list\.html\?cat=.*?" target="_blank">')
	fin = re.findall(com,html.content)
	for url in fin:
		url_lists.append("https:"+url[9:-18])
	#返回分类链接的列表
	return url_lists

"""




#sql = "INSERT INTO jds (title,link,sku_id,price,comnum,shop_name,shop_grade) VALUES ('titles---------','links','sku_ids','prices','comnums','shop_names','shop_grades')"
#print cur.execute(sql)
#sql = """
#		INSERT INTO jds (title,link,sku_id,price,comnum,shop_name,shop_grade) 
#		VALUES (%s,%s,%s,%s,%s,%s,%s)"""
#conn.commit()
#value = ('titles---------','links','sku_ids','prices','comnums','shop_names','shop_grades')
#cur.execute(sql,('titles---------','links','sku_ids','prices','comnums','shop_names','shop_grades'))
#conn.commit()